{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "\n",
    "seed = 1337\n",
    "\n",
    "torch.manual_seed(seed)\n",
    "torch.cuda.manual_seed_all(seed)\n",
    "np.random.seed(seed)\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Example 3-1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Perceptron(nn.Module):\n",
    "    \"\"\"\n",
    "    A perceptron is one linear layer \n",
    "    \"\"\"\n",
    "    \n",
    "    def __init__(self, input_dim):\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            input_dim (int): size of the input features\n",
    "        \"\"\"\n",
    "        super(Perceptron, self).__init__()\n",
    "        self.fc1 = nn.Linear(input_dim, 1)\n",
    "\n",
    "    def forward(self, x_in):\n",
    "        \"\"\"The forward pass of the perceptron\n",
    "\n",
    "        Args:\n",
    "            x_in (torch.Tensor): an input data tensor\n",
    "                x_in.shape should be (batch, num_features)\n",
    "        Returns:\n",
    "            the resulting tensor. tensor.shape should be (batch,).\n",
    "        \"\"\"\n",
    "        return torch.sigmoid(self.fc1(x_in)).squeeze()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Example 3-2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAH9RJREFUeJzt3Xl4leWd//H3N/seliRsSQhIWIK4EVl0qrZuiFa7jYqiUhfs1draqdW69Od0tNOptdPW6dhaxKk7FrU61FKpWitaAQHZtxDCkrBlI/t6knv+SPQXEcwBTvKc5fO6Lq+c85yH5HOuJB/v3Oc+92POOUREJLxEeR1AREQCT+UuIhKGVO4iImFI5S4iEoZU7iIiYUjlLiIShlTuIiJhSOUuIhKGVO4iImEoxqsvnJGR4fLy8rz68iIiIWn16tWVzrnM3s7zrNzz8vJYtWqVV19eRCQkmdluf87TtIyISBhSuYuIhCGVu4hIGOq13M3sf8ys3Mw2HuVxM7P/MrNiM1tvZmcEPqaIiBwLf0buTwIzPuPxS4D87v/mAr898VgiInIiei1359xSoPozTrkCeNp1WQ4MMLNhgQooIiLHLhBz7iOA0h73y7qPiYiIR/p1nbuZzaVr6obc3Nz+/NIiIn2us9NR3+qjrrmd+hYfDa0+6lvaaWjtut3Y6qOhtYPzx2dxas6APs0SiHLfC+T0uJ/dfexTnHPzgHkAhYWFuniriAQt5xx1zT4qGlqpbGilqqGNqsauj4ea2jjU1E5NU9ft2uZ2apq6Styfy1JnpcaHRLkvAm4zsxeAqUCtc25/AD6viEifcM5R2dDG3ppm9h5qZn9tM/trWzhQ28KBuhYO1rVQXt9Km6/ziP8+PTGWgUmxDEyOIzMlnvysVNITY0lLjCUtIebjj6kJsaTEx5AcH0NqQtfHpNhooqKsz59jr+VuZguA84AMMysD/hWIBXDOPQYsBmYCxUAT8PW+Cisi4q/OTse+2mZ2Vjayq7KRkspG9lQ1sae6idJDTbS0f7K4E2OjGTYggSGpCRSOHMiQtAQyU+PJTI0nIyWewSlxDE6OZ2BSLDHRwf8WoV7L3Tk3q5fHHfCtgCUSETkGzjnK61vZsr+OrQfqKTpQz/byBorLG2hu7/j4vKS4aHIHJTEqI5lzx2aSPTCREQOTGDEgkREDEklLjMGs70fU/cWzjcNERI5HeV0La0prWFdaw8Z9dWzeV0tlQ9vHjw9LT2BMVgpXT8lhTFYKJ2WmMCojmazU+LAq796o3EUkaHV2OrYeqGfV7mpW7jrE6l3V7KttASAmysgfksrnx2UxcXgaE4alMX5oGulJsR6nDg4qdxEJGs45dlY28u72SpbtqGL5zipqmtoBGJqWQGHeQG7KHchpOelMHJ5OQmy0x4mDl8pdRDzV0t7Bsh1V/G1rOX8vKqe0uhmAEQMSuXDCEKafNJgz8waRPTAxoqZVTpTKXUT6XUOrj7e2HOT1jQd4p6iCprYOkuKiOeukDOaecxLn5GcwcnCy1zFDmspdRPpFS3sHf9tazqK1+3h7Wzmtvk6yUuP50ukjuLBgCGedNJj4GE2zBIrKXUT6jHOOtaU1vLi6jNfW7aOuxUdmajyzpuRy6SnDmJw7sF/e0BOJVO4iEnD1Le28unYfz6/Yw5b9dSTGRjPj5KF85YwRnHVSBtEq9D6ncheRgNld1ciT7+/ixVVlNLT6mDg8jX//8slcfupwUhO0RLE/qdxF5IStLa3hN28X88aWg8REGZedMpzrp4/ktJwBWuHiEZW7iBy3ZTuqePTtYt4rriQ9MZZvnTeG66aPZEhagtfRIp7KXUSO2Yd7DvHzJdt4f0cVmanx3DtzPNdMHUlKvColWOg7ISJ+236wnp/+ZStvbS0nIyWO+y8r4JqpuXqnaBBSuYtIr6oaWvnVm9t5/oM9JMVFc+fF45hzVh7JGqkHLX1nROSoOjodzy7fzc//uo2mtg6unZrL7efnMzgl3uto0guVu4gc0drSGn746gY27q3jn8Zk8K9fLCB/SKrXscRPKncR+YSmNh8PL9nGk+/vIjMlnl/POp3LThmmJY0hRuUuIh9btqOKH7y8nj3VTcyelssPZozXm49ClMpdRGj1dfDw69uY/95OcgclseCWaUw/abDXseQEqNxFItz2g/V8e8Eath6o57ppI7ln5niS4lQNoU7fQZEI5ZzjDytL+ddFm0iJj+GJGwo5f8IQr2NJgKjcRSJQc1sHP3x1Iy9/WMY/jcngF1edSlaqtgwIJyp3kQizs7KRbzyzmqLyem4/P5/vnJ+vLXjDkMpdJIK8u72Cbz33IdFRxpNfn8K5YzO9jiR9ROUuEgGcczz5/i5+/OctjMlMYf4NheQMSvI6lvQhlbtImPN1dPKjP23i2eV7uLBgCL+86jTt3hgB9B0WCWNNbT6+s2ANb24p59ZzR/ODi8frmqURQuUuEqYqG1q56alVbCir4cErJnLd9DyvI0k/UrmLhKF9Nc3Mnr+CfbXNPDZ7MhdNHOp1JOlnKneRMLOzspHZ81dQ19zOMzdN5cy8QV5HEg+o3EXCyNYDdcye/wGdzrFg7jROHpHudSTxSJQ/J5nZDDPbZmbFZnb3ER7PNbO3zWyNma03s5mBjyoin2XrgTqueXwF0VGw8FYVe6TrtdzNLBp4FLgEKABmmVnBYaf9EFjonDsduBr4TaCDisjRfVTscdFR/GHudMZk6aIakc6fkfsUoNg5V+KcawNeAK447BwHpHXfTgf2BS6iiHyWbQfquebxFcRGGwvmTiMvI9nrSBIE/JlzHwGU9rhfBkw97JwfAX81s28DycAFAUknIp9pZ2Uj187vKvYX5k5nlIpduvk15+6HWcCTzrlsYCbwjJl96nOb2VwzW2VmqyoqKgL0pUUi00fLHTud47mbp6nY5RP8Kfe9QE6P+9ndx3q6CVgI4JxbBiQAGYd/IufcPOdcoXOuMDNTGxaJHK/KhlZmP9G13PHpG6cwJivF60gSZPwp95VAvpmNMrM4ul4wXXTYOXuA8wHMbAJd5a6huUgfaGj1Mef3H7Cvppkn5pypVTFyRL2Wu3POB9wGLAG20LUqZpOZPWBml3efdgdwi5mtAxYAc5xzrq9Ci0Sq9o5Ovvnch2zZX89vrj2DKaP0BiU5Mr/exOScWwwsPuzY/T1ubwbODmw0EenJOce9f9zA0qIKHvrqJL4wXpfEk6ML1AuqItLHfvXmdl5cXcbt5+dz1Zm5XseRIKdyFwkBr67ZyyNvbefKwmy+e0G+13EkBKjcRYLc6t2HuOvl9UwbPYgff2kSZtqPXXqnchcJYmWHmrj1mVUMT0/gt9dOJi5Gv7LiH+0KKRKkGlt93PzUKlp9nbww90wGJsd5HUlCiIYBIkHIOcddL62n6GA9j15zht6kJMdM5S4ShH63tIQ/b9jPD2aM55yxeje3HDuVu0iQWVpUwc9e38plpwxj7jmjvY4jIUrlLhJESqub+PaCNYwdksrPvnaKVsbIcVO5iwSJVl8Htz3/IZ2djsdmTyYpTusd5Pjpp0ckSPzkz1tYV1bLY7Mn64IbcsI0chcJAn9at4+nlu3m5n8axYyTh3odR8KAyl3EYzsrG7n75fVMHjmQH1wy3us4EiZU7iIeavV18O0FHxIbE8WvZ51ObLR+JSUwNOcu4qGfvb6NjXvrmHfdZIYPSPQ6joQRDRNEPPK3rQd54r2d3DB9JBdN1Dy7BJbKXcQD5XUtfP/F9UwYlsY9Myd4HUfCkMpdpJ855/j+S+tpavPx61mnkRAb7XUkCUMqd5F+9vSy3SwtquC+SwsYk5XqdRwJUyp3kX60/WA9P1m8hc+Py2T2VF0qT/qOyl2kn7T5Orn9hbWkxMfws6+dqn1jpE9pKaRIP3nkrSI2769j/vWFZKbGex1HwpxG7iL9YM2eQ/z27zu4sjCbCwqGeB1HIoDKXaSPNbd1cMfCdQxLT+T/XVbgdRyJEJqWEeljP1uylZLKRp6/eSqpCbFex5EIoZG7SB9aXlLF7/+xizln5XHWmAyv40gEUbmL9JGmNh93vbSekYOTuGvGOK/jSITRtIxIH3l4yTb2VDfxwtxpuqqS9DuN3EX6wKpd1Tz5/i5umD6SaaMHex1HIpDKXSTAWto7uPOl9WQPTOSuGbr4hnhDfyuKBNgv3yhiZ/fqmOR4/YqJN/wauZvZDDPbZmbFZnb3Uc650sw2m9kmM3s+sDFFQsOGsloef7eEq8/M0eoY8VSvwwoziwYeBS4EyoCVZrbIObe5xzn5wD3A2c65Q2aW1VeBRYJVe0cnd728noyUeO3RLp7zZ+Q+BSh2zpU459qAF4ArDjvnFuBR59whAOdceWBjigS/eUtL2LK/jge/dDLpiXqzknjLn3IfAZT2uF/WfaynscBYM/uHmS03sxmBCigSCnZUNPDIW9uZOWkoF+uSeRIEAvVqTwyQD5wHZANLzWySc66m50lmNheYC5Cbq72sJTx0djru+eMGEmKi+NHlE72OIwL4N3LfC+T0uJ/dfaynMmCRc67dObcTKKKr7D/BOTfPOVfonCvMzMw83swiQeXF1aV8sLOae2dOICs1wes4IoB/5b4SyDezUWYWB1wNLDrsnFfpGrVjZhl0TdOUBDCnSFCqqG/l3/+8hSmjBnFlYU7v/0Ckn/Ra7s45H3AbsATYAix0zm0yswfM7PLu05YAVWa2GXgbuNM5V9VXoUWCxQOvbaalvZOffHkSUVG6spIED7/m3J1zi4HFhx27v8dtB3yv+z+RiPD3beX8ad0+/uWCsYzJSvE6jsgnaPsBkePQ3NbBD1/dyEmZyXzjvNFexxH5FL03WuQ4PPLWdsoONfOHudOIj4n2Oo7Ip2jkLnKMth6oY/67JVxZmM1U7fgoQUrlLnIMOjsd9/5xA2mJsdxzibYYkOClchc5BgtW7uHDPTXcN3MCA5PjvI4jclQqdxE/VdS38tBftjJ99GC+csbhO3CIBBeVu4iffrJ4Cy3tnfz4yydjpjXtEtxU7iJ+eH9HJa+s2cs3zh3NSZla0y7BT+Uu0otWX9ea9txBSXzz82O8jiPiF61zF+nFvHdKKKlo5Mmvn0lCrNa0S2jQyF3kM+yuauTXbxdz6aRhnDdOFxiT0KFyFzkK5xz3/+8m4qKjuP+LBV7HETkmKneRo1i84QDvFFXwvQvHMiRN+7RLaFG5ixxBfUs7D7y2iYJhaVw/faTXcUSOmV5QFTmCX76xnfL6Vh6bPZmYaI2BJPTop1bkMBv31vLk+zuZNSWX03MHeh1H5Lio3EV66Ox0/PDVjQxMiuMHF4/3Oo7IcVO5i/TwwspS1pbWcN+lE0hPivU6jshxU7mLdKtsaOWh17cybfQgvny6NgaT0KZyF+n2H4u30tTm48df0sZgEvpU7iLAsh1VvPxhGbd8bjRjslK9jiNywlTuEvHafJ388NUN5AxK5NtfyPc6jkhAaJ27RLzH3y1hR0Ujv59zJolx2hhMwoNG7hLR9lQ18V9vbWfmpKF8frw2BpPwoXKXiOWc4/5FG4mJMu6/bKLXcUQCSuUuEWvxhgP8fVsF37toHEPTtTGYhBeVu0SkupZ2fvSnTZw8Io0btDGYhCG9oCoR6eHXt1HV0MoTNxRqYzAJS/qploizZs8hnl2xm+un53FK9gCv44j0CZW7RJT2jk7ufWUjWanx3HHRWK/jiPQZTctIRHnivZ1s2V/Hb689g9QEbQwm4cuvkbuZzTCzbWZWbGZ3f8Z5XzUzZ2aFgYsoEhh7qpr41ZtFXFgwhBknD/U6jkif6rXczSwaeBS4BCgAZpnZp64WbGapwO3AikCHFDlRzjnue3UD0WY8cMVEbQwmYc+fkfsUoNg5V+KcawNeAK44wnkPAg8BLQHMJxIQ/7t2H+9ur+SuGeMZlp7odRyRPudPuY8ASnvcL+s+9jEzOwPIcc79OYDZRAKiurGNB1/bzGk5A5g9TWvaJTKc8AuqZhYF/AKY48e5c4G5ALm5uSf6pUX88uBrm6ltbue5r04iOkrTMRIZ/Bm57wVyetzP7j72kVTgZODvZrYLmAYsOtKLqs65ec65QudcYWZm5vGnFvHT37eV88qavXzzvJMYPzTN6zgi/cafcl8J5JvZKDOLA64GFn30oHOu1jmX4ZzLc87lAcuBy51zq/oksYifGlp93PfKRsZkpfCtL4zxOo5Iv+q13J1zPuA2YAmwBVjonNtkZg+Y2eV9HVDkeP18yTb21Tbz0FcnER+jfdolsvg15+6cWwwsPuzY/Uc597wTjyVyYlbtquapZbu4ftpIJo8c5HUckX6n7Qck7LS0d3DXS+sZnp7IXTPGex1HxBPafkDCzi/eKKKkspHnbp5Kcrx+xCUyaeQuYeXDPYeY/24Js6bkcvaYDK/jiHhG5S5h46PpmKFpCdw7U9MxEtn0N6uEjV++UURxeQNP3ThFOz5KxNPIXcLCql3VzOuejjl3rN4gJ6Jyl5DX1ObjjhfXkT0wkfsuneB1HJGgoGkZCXk//ctW9lQ3seCWaaRodYwIoJG7hLilRRU8vWw3N549immjB3sdRyRoqNwlZB1qbOP7L64jPyuFOy8e53UckaCiv2ElJDnnuOePGzjU1Mbvv34mCbHaO0akJ43cJSS9uLqM1zcd4PsXjWPi8HSv44gEHZW7hJzdVY3826JNTBs9iJs/N9rrOCJBSeUuIaXN18l3FqwhOsr4zytP05WVRI5Cc+4SUv7zr9tYV1bLb689gxEDdKFrkaPRyF1CxjtFFfxuaQnXTM3lkknDvI4jEtRU7hISyutbuGPhWsYNSeX+ywq8jiMS9DQtI0Gvo9Nx+4K1NLT6eP6WaVr2KOIHlbsEvV++UcSykioe/topjB2S6nUckZCgaRkJam9vK+e/3y7mysJs/rkwx+s4IiFD5S5Ba29NM//yh7WMH5rKA1ec7HUckZCicpeg1NLewTeeWY2vw/Hb2ZM1zy5yjDTnLkHHOcd9r2xkw95aHr++kFEZyV5HEgk5GrlL0Hl62W5e/rCM716Qz4UFQ7yOIxKSVO4SVFaUVPHga5u5YMIQvvOFfK/jiIQslbsEjd1VjXzj2dXkDk7iF1edSpT2jRE5bip3CQp1Le3c9NQqOh08ccOZpCXEeh1JJKSp3MVzvo5Obnt+DbsqG3ls9mS9gCoSAFotI55yzvHAa5tZWlTBT78yiekn6TqoIoGgkbt46rF3Snh62W7mnjOaq6fkeh1HJGyo3MUzr67Zy0Ovb+WLpw7n7hnjvY4jElb8Knczm2Fm28ys2MzuPsLj3zOzzWa23szeMrORgY8q4eQfxZXc+dI6po0exM//+RStjBEJsF7L3cyigUeBS4ACYJaZHb6h9hqg0Dl3CvAS8LNAB5Xwsba0hrlPr2J0Rgq/u66Q+BhtLSASaP6M3KcAxc65EudcG/ACcEXPE5xzbzvnmrrvLgeyAxtTwsW2A/XM+f0HDE6J5+mbppCeqCWPIn3Bn3IfAZT2uF/WfexobgL+ciKhJDztrmpk9hMriIuO4rmbpzIkLcHrSCJhK6BLIc1sNlAInHuUx+cCcwFyc7UyIpKUVjdxzeMraO/oZOGt08kZlOR1JJGw5s/IfS/Q8yoJ2d3HPsHMLgDuAy53zrUe6RM55+Y55wqdc4WZmZnHk1dCUNmhJmY9vpz6lnaeuXGqrqYk0g/8KfeVQL6ZjTKzOOBqYFHPE8zsdOB3dBV7eeBjSqgqO9TE1fOWU9fcznM3T2NSdrrXkUQiQq/l7pzzAbcBS4AtwELn3CYze8DMLu8+7WEgBXjRzNaa2aKjfDqJILurGj8u9mdvnqpiF+lHfs25O+cWA4sPO3Z/j9sXBDiXhLhtB+q57omuOXaN2EX6n/aWkYBbV1rDDb//gPiYKBbeOp18zbGL9DuVuwTUO0UVfPPZ1QxKieO5m6aRO1irYkS8oL1lJGAWrizlxidXkjs4mZe+cZaKXcRDGrnLCXPO8chb2/nVm9v5XH4Gv7n2DFJ1sQ0RT6nc5YQ0t3Vw50vreG39fr42OZv/+MokYqP1B6GI11Tuctz21TQz95lVbNpXx92XjOfWc0Zjpt0dRYKByl2Oy/KSKm57fg0t7R3Mv76Q8ycM8TqSiPSgcpdj0tnp+N3SEh5espW8wck8f4u2ExAJRip38duhxjbufGkdb24p59JThvHQV08hJV4/QiLBSL+Z4pf3tldyx4trqW5s40dfLOCGs/I0vy4SxFTu8pla2jv4+ZJtzH9vJ2OyUvifOWcycbi2EhAJdip3OarVuw9x10vr2FHRyHXTRnLvzAkkxumSeCKhQOUun9LU5uMXfy3iiX/sZHh6Ik/fOIVzxmr/fZFQonKXT/jrpgP82582s7emmWun5nL3JeP1blOREKRyF6Br7/UHX9vMm1vKGTcklYW3TmfKqEFexxKR46Ryj3C1Te38+m/beWrZLmKjo7hv5gTmnJ2nLQREQpzKPUK1tHfw7PLdPPp2MTXN7Vw5OYc7LhpLVlqC19FEJABU7hGmzdfJwlWl/Ppv2zlY18rn8jO455IJFAxP8zqaiASQyj1CNLd18MLKPcxbWsL+2hYKRw7kkatPZ9rowV5HE5E+oHIPc1UNrTy3Yg9Pvb+LqsY2puQN4j++Molzx2bqHaYiYUzlHqY276vjqfd38cravbT5OjlvXCbfPG+MVsCIRAiVexhpbuvgT+v38fyKPawtrSEhNoorC7OZc9YoxmSleB1PRPqRyj3EdXY6PthVzcury/jLxgM0tPoYk5XC/ZcV8JUzRjAgKc7riCLiAZV7CHLOsa6slj+v38fiDQfYW9NMclw0MycN42uTs5kyapDm00UinMo9RLR3dPLBzmre2HyQNzYfZG9NM7HRxjn5mdx58TgunjhUm3qJyMdU7kFsX00zS4sqeKeogveKK6lv8REfE8Xn8jO4/YJ8Li4YSnqS9n0RkU9TuQeRA7UtrNxVzbKSKpbtqGJnZSMAw9ITmHnyML4wIYvP5WeQFKdvm4h8NrWER9p8nWw9UMfa0hrW7Klh1e5qSqubAUiNj2HKqEFcOzWXc8Zmkp+Vojl0ETkmKvd+0NDqY9uBerYeqGPj3jo27atl6/562jo6AchIiadw5EBumJ7HmXmDmDg8jRht3CUiJ0DlHiDOOaob29hZ2UhJRSPFFQ0Ulzewvbz+4xE5QHpiLBOHpzHn7DxOzR7AqTnpjBiQqJG5iASUyv0YNLb62FfTTFlNM3sPNVN2qJnS6ib2VDexu6qRuhbfx+fGRUcxOjOZU7MHcFVhDuOHpjFuaCrZA1XkItL3Ir7cOzsdtc3tVDW2UdXQSmVDGxX1LVQ0tHKwrpWDdS0crGthf20L9T3KGyA22sgZmETOoCROyxlAXkYyozOSyctIJmdgoqZWRMQzfpW7mc0AHgGigfnOuZ8e9ng88DQwGagCrnLO7Qps1CNzztHq66Sh1Udjq4/6Fh8NrT4aWnzUtbRT3+Kjrrmd2uZ2aj762NTGoab//7Gj033q80ZHGVmp8WSlxjNycDLTRw9maHoiwwckMGJAIiMGJpKVmkB0lEbhIhJ8ei13M4sGHgUuBMqAlWa2yDm3ucdpNwGHnHNjzOxq4CHgqr4IvHBlKY8t3UFTaweNbT6a2jqOWM6HS4qLJj0xlvTEWAYkxZKflcKApDgGJ8cxKDmOwSlxDE6OJyM1joyUeAYlxRGl4haREOXPyH0KUOycKwEwsxeAK4Ce5X4F8KPu2y8B/21m5pzrvXWP0cDkOAqGpZEUF01SXAxJcdEkx8eQEh9DcnwMqQkxpMbHkJIQQ1pCLGmJsaTExxAXoykSEYkc/pT7CKC0x/0yYOrRznHO+cysFhgMVPY8yczmAnMBcnNzjyvwhQVDuLBgyHH9WxGRSNGvw1nn3DznXKFzrjAzM7M/v7SISETxp9z3Ajk97md3HzviOWYWA6TT9cKqiIh4wJ9yXwnkm9koM4sDrgYWHXbOIuCG7ttfA/7WF/PtIiLin17n3Lvn0G8DltC1FPJ/nHObzOwBYJVzbhHwBPCMmRUD1XT9D0BERDzi1zp359xiYPFhx+7vcbsF+OfARhMRkeOl9YEiImFI5S4iEoZU7iIiYci8WtRiZhXAbk+++InJ4LA3Z0WISHzees6RI5Se90jnXK9vFPKs3EOVma1yzhV6naO/ReLz1nOOHOH4vDUtIyIShlTuIiJhSOV+7OZ5HcAjkfi89ZwjR9g9b825i4iEIY3cRUTCkMr9BJjZHWbmzCzD6yx9zcweNrOtZrbezF4xswFeZ+pLZjbDzLaZWbGZ3e11nr5mZjlm9raZbTazTWZ2u9eZ+ouZRZvZGjN7zessgaRyP05mlgNcBOzxOks/eQM42Tl3ClAE3ONxnj7T49KSlwAFwCwzK/A2VZ/zAXc45wqAacC3IuA5f+R2YIvXIQJN5X78fgncBUTEixbOub8653zdd5fTta9/uPr40pLOuTbgo0tLhi3n3H7n3Ifdt+vpKrsR3qbqe2aWDVwKzPc6S6Cp3I+DmV0B7HXOrfM6i0duBP7idYg+dKRLS4Z90X3EzPKA04EV3ibpF7+ia5DW6XWQQPNry99IZGZvAkOP8NB9wL10TcmElc96zs65/+0+5z66/oR/rj+zSf8wsxTgZeC7zrk6r/P0JTO7DCh3zq02s/O8zhNoKvejcM5dcKTjZjYJGAWsMzPomp740MymOOcO9GPEgDvac/6Imc0BLgPOD/MrbflzacmwY2axdBX7c865P3qdpx+cDVxuZjOBBCDNzJ51zs32OFdAaJ37CTKzXUChcy5UNh06LmY2A/gFcK5zrsLrPH2p+zrARcD5dJX6SuAa59wmT4P1IesaqTwFVDvnvut1nv7WPXL/vnPuMq+zBIrm3MVf/w2kAm+Y2Voze8zrQH2l+4Xjjy4tuQVYGM7F3u1s4DrgC93f37XdI1oJURq5i4iEIY3cRUTCkMpdRCQMqdxFRMKQyl1EJAyp3EVEwpDKXUQkDKncRUTCkMpdRCQM/R9j5Qv3/WpxFgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import torch\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "x = torch.arange(-5., 5., 0.1)\n",
    "y = torch.sigmoid(x)\n",
    "plt.plot(x.numpy(), y.detach().numpy())\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Example 3-3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAD8CAYAAABzTgP2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl8XXWd//HXJzdb9y3pnm60UsrSArEslWFrocoMZUZUdMSiMB39yTiOisI4P50fjvPAn4+R+fkYRukDkSqbUMVWKZRdZYDSFErbFErT0iVpkqZ72uz3fn5/3FO8CUm3e5Nzl/fz8bhzz/me77n53LHkne9ZvsfcHRERkaPywi5ARETSi4JBREQ6UTCIiEgnCgYREelEwSAiIp0oGEREpBMFg4iIdKJgEBGRThQMIiLSSX7YBZyKkpISnzRpUthliIhklDVr1uxx99Lj9cvIYJg0aRIVFRVhlyEiklHMbPuJ9NOhJBER6UTBICIinSgYRESkEwWDiIh0omAQEZFOUhIMZna/me02sw09bDcz+7GZVZnZOjM7L2HbQjPbHLwWpqIeERE5dakaMTwAzD/G9o8C04LXIuAnAGY2HPgucAEwG/iumQ1LUU0iInIKUnIfg7v/0cwmHaPLAuAXHn+O6GtmNtTMxgCXAc+6+z4AM3uWeMA8koq6RCT7dURjtHbEaOuIv7dHY7RF4+/tHU57LEZH1OmIxuiIOdHg1RFzYh5fPvoejTnuEHMnFrx7sPz+O/Fld3COvtNp/aij/SDeh6Df+9vxhL6J7Qm6PH554cWTGDGwKAX/n+tZX93gNg7YmbBeHbT11P4BZraI+GiDCRMm9E6VItKnojFn75FWGhpb2X+knX1Nbew/0sbB5nYONbdzsLmdw60d77+aWqM0tXfQ3BalpT1GS3uUjlj2P7fe7M/L184alzXBkDR3XwwsBigvL8/+fwkiWSAac3bua2Lb3iNs39vE9r1N7DrQTO3BZnYdbGHv4VZ6+r3evzDC4OICBhbnM7Ao/iodWET/wgj9CvPpVxChuCCP4oIIhfl5FOXnUZQfoSBiFObnURjJoyCSR37EyM87+m5Eur7MyAveI3nx5TwDw8jLgzwzjOA9aLc8MMCCbe+3B7/Aj67/efloe9CW8D0Tf+lb4kqI+ioYaoCyhPXxQVsN8cNJie0v9VFNIpJCTW0dbKg5xLrqA1TuOsSmukaqGg7T1hF7v0+/ggjjhvVjzJBipo8ezKjBRZQOKqJkYBEjBhYxfEABQ/sXMqRfAQURXTQZlr4KhuXArWb2KPETzQfdvdbMVgL/nnDC+Srgjj6qSUSScLi1g1Vb9/La1r28tnUflbsOvv/X/+jBxUwfM4iPTCth6siBTCkZwIQR/SkdWJQ2fxVLz1ISDGb2CPG//EvMrJr4lUYFAO7+U2AF8DGgCmgCPh9s22dm3wNWBx9159ET0SKSfuoPtbCyso5nN9azaus+2qIxCvPzOG/CUG69fCozy4ZyzvihlA7q3WPg0rvMPfMO15eXl7tmVxXpG81tUVZW1vHrN6p5uWoP7jC5ZABzzxjJ5dNHct6EYRQXRMIuU06Ama1x9/Lj9cuYk88i0rdqDzaz5JXtPPL6Dg42tzN+WD/+4fKpXDtrLFNHDgq7POlFCgYR6WTnvibufu5dlq/dRcyd+WeN5nMXTWL2pOHk5en8QC5QMIgIAA2NrfzXC5t5+PUd5JnxuYsm8fk5kygb3j/s0qSPKRhEclws5jz8+g5+8NQ7NLVH+dSHy/jKFdMYPaQ47NIkJAoGkRxWtbuR23+9nort+5kzdQTfW3AWU0oHhl2WhEzBIJKD3J1frd7Jd5ZX0r8wwg+vP4frzx+vewwEUDCI5Jymtg7+5bcb+M0bNXxkagl3f2qW7juQThQMIjlk14FmPv/z1by7u5Gvzp3GP1wxjYiuNJIuFAwiOeLd+kY+97PXOdLawZLPz+YvPlQadkmSphQMIjlg9bZ93PzAaooLIjz2xYs4Y8zgsEuSNKZgEMlyr23dy8L7X2fcsH784guzGT9M9yXIsSkYRLLYWzsPcPMDq5kwvD+PLrqw1x/wItlBE56LZKl36xtZ+PPXGT6wkF/efIFCQU6YgkEkC+060Mxn71tFYSSPh26+UHcxy0nRoSSRLNPSHuWLD66hqS3K0i9dxIQROqcgJyclIwYzm29mm8ysysxu72b73Wa2Nni9a2YHErZFE7YtT0U9IrnK3fmX325gXfVBfvTJmUwfrauP5OQlPWIwswhwDzAPqAZWm9lyd994tI+7/1NC/38Azk34iGZ3n5VsHSICv3h1O0vXVPOVK6dx1Zmjwy5HMlQqRgyzgSp33+rubcCjwIJj9P808EgKfq6IJHhzx36+9/uNXDl9JF+9clrY5UgGS0UwjAN2JqxXB20fYGYTgcnACwnNxWZWYWavmdl1KahHJOc0t0X52mNvMWpwMT/61Cw9UEeS0tcnn28Alrp7NKFtorvXmNkU4AUzW+/uW7ruaGaLgEUAEyZM6JtqRTLEXU+9zXt7jvDw313AkH4FYZcjGS4VI4YaoCxhfXzQ1p0b6HIYyd1rgvetwEt0Pv+Q2G+xu5e7e3lpqeZ4ETnq5c17WPLqdr4wZzIXn1YSdjmSBVIRDKuBaWY22cwKif/y/8DVRWY2HRgGvJrQNszMioLlEmAOsLHrviLSvYPN7dy29C1OKx3AN+efHnY5kiWSPpTk7h1mdiuwEogA97t7pZndCVS4+9GQuAF41N09YfczgHvNLEY8pO5KvJpJRI7tP57ZRP2hFp74X3MoLoiEXY5kiZScY3D3FcCKLm3f6bL+r93s9wpwdipqEMk1lbsO8uBr27nxwonMLBsadjmSRTQlhkgGcne+u6ySYf0L+dpVOoQkqaVgEMlAT7xZQ8X2/Xxr/nRdhSQpp2AQyTCNLe38+4p3mFk2lOvPHx92OZKFNImeSIb56R+2sOdwKz9bWK4b2aRXaMQgkkEaGlu5/+Vt/NXMsTrhLL1GwSCSQe55sYq2aIx/mqu5kKT3KBhEMkTNgWYeXrWD688bz5TSgWGXI1lMwSCSIX783GYAvqLRgvQyBYNIBnhvzxGWvlHNZy6YwLih/cIuR7KcgkEkA9zzYhUFEePLl08NuxTJAQoGkTRXe7CZZWtr+FR5GaWDisIuR3KAgkEkzd3/8nvEHG65ZErYpUiOUDCIpLGDze08vGoH15w9hrLh/cMuR3KEgkEkjT20ajtH2qL8/aUaLUjfUTCIpKmW9ij3v7yNS6aVcObYIWGXIzlEwSCSpp54s4Y9h1v54qWnhV2K5JiUBIOZzTezTWZWZWa3d7P9JjNrMLO1weuWhG0LzWxz8FqYinpEMp27s+SVbcwYM5iLTxsRdjmSY5KeXdXMIsA9wDygGlhtZsu7eUTnr9z91i77Dge+C5QDDqwJ9t2fbF0imaxi+37eqWvkrr85GzPNoCp9KxUjhtlAlbtvdfc24FFgwQnuezXwrLvvC8LgWWB+CmoSyWi/fHU7g4rzuXbW2LBLkRyUimAYB+xMWK8O2rr6uJmtM7OlZlZ2kvuK5IyGxlae2lDL9eePp3+hHpkifa+vTj7/Dpjk7ucQHxUsOdkPMLNFZlZhZhUNDQ0pL1AkXTxWsZP2qPPZCyeGXYrkqFQEQw1QlrA+Pmh7n7vvdffWYPU+4PwT3TfhMxa7e7m7l5eWlqagbJH0E405D722nTlTR3CaptaWkKQiGFYD08xsspkVAjcAyxM7mNmYhNVrgbeD5ZXAVWY2zMyGAVcFbSI56YV3drPrYAs3Xjgp7FIkhyV9ANPdO8zsVuK/0CPA/e5eaWZ3AhXuvhz4ipldC3QA+4Cbgn33mdn3iIcLwJ3uvi/ZmkQy1cOrtjNqcBFzzxgZdimSw1JyZsvdVwArurR9J2H5DuCOHva9H7g/FXWIZLL6Qy384d0GvnTZaeRHdO+phEf/+kTSxG/eqCHmcP35ZcfvLNKLFAwiacDdeXzNTj48aRiTSwaEXY7kOAWDSBp4Y8cBtjYc4frzx4ddioiCQSQdLF2zk34FEa45R3c6S/gUDCIha26L8ru3avno2aMZWKQ7nSV8CgaRkD1dWcvh1g4+oZPOkiYUDCIh+80bNZQN78cFk4eHXYoIoGAQCdXuxhb+p2oP180aR16epteW9KBgEAnRk+tqiTks0PTakkYUDCIhWrZ2FzPGDGbqyEFhlyLyPgWDSEi27z3C2p0HNFqQtKNgEAnJ8rW7APirmQoGSS8KBpEQuDvL3trF7EnDGTu0X9jliHSiYBAJwdu1jVTtPqxnOktaUjCIhGDZWzXk5xkfO3vM8TuL9DEFg0gfc3d+/1Ytl0wrYfiAwrDLEfmAlASDmc03s01mVmVmt3ez/WtmttHM1pnZ82Y2MWFb1MzWBq/lXfcVyTbrqg9Sc6BZE+ZJ2kp6xi4ziwD3APOAamC1mS13940J3d4Eyt29ycy+BPxf4FPBtmZ3n5VsHSKZYsX6WgoixrwzRoVdiki3UjFimA1UuftWd28DHgUWJHZw9xfdvSlYfQ3QpPOSk9ydJ9fXMmdqCUP6F4Rdjki3UhEM44CdCevVQVtPbgaeSlgvNrMKM3vNzK7raSczWxT0q2hoaEiuYpGQrK85SPX+Zp10lrTWp5O/m9lngXLg0oTmie5eY2ZTgBfMbL27b+m6r7svBhYDlJeXe58ULJJiT66vJT/PuGqGDiNJ+krFiKEGSJxIfnzQ1omZzQW+DVzr7q1H2929JnjfCrwEnJuCmkTSjrvz1Po6Lp5awtD+uhpJ0lcqgmE1MM3MJptZIXAD0OnqIjM7F7iXeCjsTmgfZmZFwXIJMAdIPGktkjUqdx1ix74mrjl7dNiliBxT0oeS3L3DzG4FVgIR4H53rzSzO4EKd18O/BAYCDxuZgA73P1a4AzgXjOLEQ+pu7pczSSSNZ5cX0skz7hqhoJB0ltKzjG4+wpgRZe27yQsz+1hv1eAs1NRg0g6ix9GquXi00YwTDe1SZrTnc8ifWBTfSPb9jYx/yyNFiT9KRhE+sDKDfWYwTxdjSQZQMEg0geerqzj/AnDGDmoOOxSRI5LwSDSy3bsbeLt2kM6jCQZQ8Eg0stWVtYBcPWZCgbJDAoGkV62srKOGWMGUza8f9iliJwQBYNIL9rd2MKaHfs1WpCMomAQ6UXPbqzHHZ1fkIyiYBDpRU9vqGNyyQA+NGpg2KWInDAFg0gvOdjczqtb9nLVjFEEU8GIZAQFg0gveWnTbjpizlU6vyAZRsEg0kue2VhPycAizi0bGnYpIidFwSDSC1o7orz0zm7mzRhFXp4OI0lmUTCI9IJXtuzlSFuUq87U3EiSeRQMIr3gmcp6BhRGuPi0EWGXInLSUhIMZjbfzDaZWZWZ3d7N9iIz+1WwfZWZTUrYdkfQvsnMrk5FPSJhisWc596u57LpIynKj4RdjshJSzoYzCwC3AN8FJgBfNrMZnTpdjOw392nAncDPwj2nUH8UaBnAvOB/w4+TyRjra0+QENjK1dpim3JUKkYMcwGqtx9q7u3AY8CC7r0WQAsCZaXAlda/MLuBcCj7t7q7u8BVcHniWSsZyrrKYgYl08fGXYpIqckFcEwDtiZsF4dtHXbx907gIPAiBPcVySjPLOxjgunjGBwcUHYpYickow5+Wxmi8yswswqGhoawi5HpFtVuw+zteGIntQmGS0VwVADlCWsjw/auu1jZvnAEGDvCe4LgLsvdvdydy8vLS1NQdkiqffMxvizFxQMkslSEQyrgWlmNtnMComfTF7epc9yYGGwfD3wgrt70H5DcNXSZGAa8HoKahIJxTOV9cwcP4QxQ/qFXYrIKUs6GIJzBrcCK4G3gcfcvdLM7jSza4NuPwNGmFkV8DXg9mDfSuAxYCPwNPBld48mW5NIGOoPtbB25wHNjSQZLz8VH+LuK4AVXdq+k7DcAnyih32/D3w/FXWIhOnZjfUAukxVMl7GnHwWSXfPbqxncskApo7UsxcksykYRFKgsaWdV7bsYZ6evSBZQMEgkgIvbWqgPeo6jCRZQcEgkgLxZy8Ucu6EYWGXIpI0BYNIkto6Yrz0zm7mnjGKiJ69IFlAwSCSpFe27KGxtUPPXpCsoWAQSdLKyjoGFuVz8WklYZcikhIKBpEkRGPOM5X1XHZ6KcUFmjFesoOCQSQJa7bvZ++RNuafpbudJXsoGESS8PSGOgrz87jsdD17QbKHgkHkFLk7Kyvr+MjUEgYWpWR2GZG0oGAQOUWVuw5Rc6CZ+Zo0T7KMgkHkFK2srCPP4MozdBhJsouCQeQUPb2hjtmThzNiYFHYpYiklIJB5BRsaTjM5t2HuVqHkSQLKRhETsGKdbUAukxVslJSwWBmw83sWTPbHLx/YAYxM5tlZq+aWaWZrTOzTyVse8DM3jOztcFrVjL1iPSVJ9fXct6EoXqEp2SlZEcMtwPPu/s04Plgvasm4HPufiYwH/hPMxuasP02d58VvNYmWY9Ir9vacJh36hr52Nljwi5FpFckGwwLgCXB8hLguq4d3P1dd98cLO8CdgOlSf5ckdA8taEOQMEgWSvZYBjl7rXBch1wzOklzWw2UAhsSWj+fnCI6W4z0+UdkvaeXFfLuROGMnaoDiNJdjpuMJjZc2a2oZvXgsR+7u6AH+NzxgC/BD7v7rGg+Q5gOvBhYDjwrWPsv8jMKsysoqGh4fjfTKQXbNtzhI21h7hGowXJYse9j9/d5/a0zczqzWyMu9cGv/h399BvMPAk8G13fy3hs4+ONlrN7OfAN45Rx2JgMUB5eXmPASTSm1Zs0NVIkv2SPZS0HFgYLC8ElnXtYGaFwBPAL9x9aZdtY4J3I35+YkOS9Yj0qhXra5lZNpTxw/qHXYpIr0k2GO4C5pnZZmBusI6ZlZvZfUGfTwJ/AdzUzWWpD5nZemA9UAL8W5L1iPSaHXub2FBziGvO1mhBsltSU0K6+17gym7aK4BbguUHgQd72P+KZH6+SF/63bpdgK5GkuynO59FToC789s3a/jwpGE6jCRZT8EgcgLeqWtk8+7DXDtrXNiliPQ6BYPICVi2dhf5eabLVCUnKBhEjiMWc3731i4umVbC8AGFYZcj0usUDCLH8caO/dQcaGaBDiNJjlAwiBzHsrW7KC7IY96MY874IpI1FAwix9AejfHk+lrmnjGKAUVJXd0tkjEUDCLH8PLmPew70sa1M8eGXYpIn1EwiBzD42t2MnxAIZedPjLsUkT6jIJBpAf7j7Tx3MbdXDdrHIX5+k9Fcof+tYv0YNnaGtqiMT5RPj7sUkT6lIJBpAePr6nmrHGDOWPM4LBLEelTCgaRbmzcdYjKXYf4xPllYZci0ucUDCLdeHzNTgojeboaSXKSgkGki7aOGMvW7mLujJEM0xQYkoMUDCJdPP92PfuOtOkwkuSspILBzIab2bNmtjl4H9ZDv2jC09uWJ7RPNrNVZlZlZr8KHgMqEqoHV21n7JBiLplWEnYpIqFIdsRwO/C8u08Dng/Wu9Ps7rOC17UJ7T8A7nb3qcB+4OYk6xFJStXuw/xP1V7+9sKJ5Ec0oJbclOy//AXAkmB5CXDdie5oZgZcASw9lf1FesNDq7ZTEDE+Wa7DSJK7kg2GUe5eGyzXAT1NP1lsZhVm9pqZHf3lPwI44O4dwXo1oHmNJTRNbR0sXVPNR88aQ+mgorDLEQnNcaeLNLPngNHdbPp24oq7u5l5Dx8z0d1rzGwK8IKZrQcOnkyhZrYIWAQwYcKEk9lV5IQsW7uLxpYOPnfRxLBLEQnVcYPB3ef2tM3M6s1sjLvXmtkYYHcPn1ETvG81s5eAc4FfA0PNLD8YNYwHao5Rx2JgMUB5eXlPASRyStydX766nemjB3H+xG6voRDJGckeSloOLAyWFwLLunYws2FmVhQslwBzgI3u7sCLwPXH2l+kL7yx4wAbaw9x40UTiZ/+EsldyQbDXcA8M9sMzA3WMbNyM7sv6HMGUGFmbxEPgrvcfWOw7VvA18ysivg5h58lWY/IKfnZy1sZVJzPdXp8p8jxDyUdi7vvBa7spr0CuCVYfgU4u4f9twKzk6lBJFnb9hzhqQ11fOnS0/SUNhF057MIi/+0lYJIHjfNmRR2KSJpQcEgOa2hsZWla6r5+HnjGTmoOOxyRNKCgkFy2pJXttEejfF3l0wOuxSRtKFgkJx1uLWDX7y6jatnjGZK6cCwyxFJGwoGyVmPrNrBoZYO/v7SKWGXIpJWFAySkw63dvCTP2zhI1NLOHeCbmgTSaRgkJz085ffY9+RNr5x9elhlyKSdhQMknMONLWx+E9bmTdjFLPKhoZdjkjaUTBIzrn3j1s53NrB16/6UNiliKQlBYPklN2NLTzwP9v4q3PGMn304LDLEUlLCgbJKT9+fjNt0Rj/NE+jBZGeKBgkZ1TuOsjDq3bwtxdMYHLJgLDLEUlbCgbJCe7Od5ZVMqx/IV+fpyuRRI5FwSA54Yk3a1izfT/fmj+dIf0Lwi5HJK0pGCTrHWpp599XvMOssqFcf/74sMsRSXuafF6y3o+eeZe9R1q5/6Zy8vL0dDaR40lqxGBmw83sWTPbHLx/YG4BM7vczNYmvFrM7Lpg2wNm9l7CtlnJ1CPS1Stb9vDAK9u48cKJnDNeN7OJnIhkDyXdDjzv7tOA54P1Ttz9RXef5e6zgCuAJuCZhC63Hd3u7muTrEfkfY0t7dz2+Domlwzg9o9OD7sckYyRbDAsAJYEy0uA647T/3rgKXdvSvLnihzXnb/bSO3BZv7jkzPpX6ijpiInKtlgGOXutcFyHTDqOP1vAB7p0vZ9M1tnZnebWVFPO5rZIjOrMLOKhoaGJEqWXPBMZR2Pr6nmS5edxnmaPVXkpBw3GMzsOTPb0M1rQWI/d3fAj/E5Y4CzgZUJzXcA04EPA8OBb/W0v7svdvdydy8vLS09XtmSw3bsbeK2peuYMWYw/3il7nAWOVnHHV+7+9yetplZvZmNcffa4Bf/7mN81CeBJ9y9PeGzj442Ws3s58A3TrBukW41tXWw6JcVuDs/+ex5FObrimyRk5XsfzXLgYXB8kJg2TH6fpouh5GCMMHMjPj5iQ1J1iM5zN355tJ1bKpv5MefPpeJIzTthcipSDYY7gLmmdlmYG6wjpmVm9l9RzuZ2SSgDPhDl/0fMrP1wHqgBPi3JOuRHHbvH7fy+3W13Hb16Vx2+siwyxHJWEldquHue4Eru2mvAG5JWN8GjOum3xXJ/HyRo5auqeaup97hmnPG8KVLTwu7HJGMpgOwkvGe3lDLN5e+xSXTSvjRJ2cSPzIpIqdKwSAZ7U+bG/jKI2uZVTaUe288n6L8SNgliWQ8BYNkrGcq67hlSQVTSgfw85tm6yY2kRRRMEhGeuT1HXzxwTVMHzOYh//uQk2lLZJC+hNLMkos5vz4hc3853Obuez0Uv77b8/TSEEkxfRflGSMA01tfP2xt3j+nd18/Lzx3PXxsymIaNArkmoKBskIa3ce4MsPvcHuxhb+z7Vn8rmLJurqI5FeomCQtNbSHuXHz29m8R+3MmpwMUu/eDEzy/RcBZHepGCQtPXKlj3882/Ws21vE9efP57/fc0MnWQW6QMKBkk7m+oa+eHKTTz3dj0TR/TnoVsuYM7UkrDLEskZCgZJG5vqGrn3D1t4Ym0NAwvzue3q0/nCnMn0K9RNayJ9ScEgoYrGnD9ubuD+l9/jT5v3UFyQx6JLpvDFS09j2IDCsMsTyUkKBgnF5vpGfvNmDb99s4bagy2MHFTEbVefzmdmT1AgiIRMwSB9oj0aY+3OAzz3dj3PbaxnS8MRInnGpR8q5Z8/dgZXnzlaD9URSRMKBukVR1o72FBzkDd2HOC1rXup2LaPI21R8vOMC6eM4MYLJ3LNOWMpHdTjY75FJCRJBYOZfQL4V+AMYHbwHIbu+s0H/h8QAe5z96MP9JkMPAqMANYAN7p7WzI1Sd9qj8ao3t/Mlt2H2VTfyOb6RjbWHqJq92FiwRPAp44cyF+fN46LppRwyYdKGFysS05F0lmyI4YNwN8A9/bUwcwiwD3APKAaWG1my919I/AD4G53f9TMfgrcDPwkyZokBdyd5vYo+460sedwGw2NrTQ0tlJ3qIVdB5qpPdjMzn3N1BxoJno0AYCxQ4o5ffQg5p81hpnjhzCzbCglAzUqEMkkyT7B7W3geFMTzAaq3H1r0PdRYIGZvQ1cAXwm6LeE+OhDwdCFuxPz+BU80ZgTdScadTpiMTpiTns0Rkew3tYRX2+PxmjtiNHWEX9v7YjS0h6lpT1GU1uU5rYOmtujHG6Ncri1gyOtHRxqbudQSzuHmjvY39RGa0fsA7WYwahBxYwZWsw544ewYNZYJo4YwOSS/kwbNUijAZEs0BfnGMYBOxPWq4ELiB8+OuDuHQntH3j8Zyp9+4n1rHpv3/vr7t5tP+9hxRP2Sexz9GMcxz1h3T3YJ2Hb0fX3t8V/6cc8vj3mHrziM4lGg/be0L8wwoCifAYW5TOgKMLg4gKmlAxkUHE+wwcUMmxAIcP6F1AysIjSQfFXycAiTVwnkuWOGwxm9hwwuptN33b3Zakvqcc6FgGLACZMmHBKnzF2aD9OHzWoywf38PM6/+xO7UdXu+tjwf8xDLM/939/PeiQZ39uj+TZ+8t5RrDeeTliRiQPInl5RPIgPy+P/Eh834JgOT+SR2HEKIjkURDJozA/j6L8+HtxQST+ys+jf2E+xQV5moRORLp13GBw97lJ/owaoCxhfXzQthcYamb5wajhaHtPdSwGFgOUl5ef0t/QX7586qnsJiKSU/rimMBqYJqZTTazQuAGYLnHj8m8CFwf9FsI9NkIREREupdUMJjZX5tZNXAR8KSZrQzax5rZCoBgNHArsBJ4G3jM3SuDj/gW8DUzqyJ+zuFnydQjIiLJs55OwKaz8vJyr6jo9pYJERHpgZmtcffy4/XT5SUiItKJgkFERDpRMIiISCcKBhER6UTBICIinWTkVUlm1gBsD7uOU1AC7Am7iD6Wi98ZcvN75+J3hsz63hPdvfR4nTIyGDKVmVWcyKVi2SQXvzPk5vfOxe8M2fm9dShJREQ6UTCIiEgmR4PAAAACd0lEQVQnCoa+tTjsAkKQi98ZcvN75+J3hiz83jrHICIinWjEICIinSgYQmJmXzczN7OSsGvpbWb2QzN7x8zWmdkTZjY07Jp6i5nNN7NNZlZlZreHXU9fMLMyM3vRzDaaWaWZ/WPYNfUVM4uY2Ztm9vuwa0klBUMIzKwMuArYEXYtfeRZ4Cx3Pwd4F7gj5Hp6hZlFgHuAjwIzgE+b2Yxwq+oTHcDX3X0GcCHw5Rz53gD/SPxxAllFwRCOu4Fv0uXx0tnK3Z9JeLb3a8Sf1peNZgNV7r7V3duAR4EFIdfU69y91t3fCJYbif+i7NXnt6cDMxsPXAPcF3YtqaZg6GNmtgCocfe3wq4lJF8Angq7iF4yDtiZsF5NDvyCTGRmk4BzgVXhVtIn/pP4H3ixsAtJteM+81lOnpk9B4zuZtO3gX8mfhgpqxzrO7v7sqDPt4kfdnioL2uTvmFmA4FfA19190Nh19ObzOwvgd3uvsbMLgu7nlRTMPQCd5/bXbuZnQ1MBt4yM4gfUnnDzGa7e10flphyPX3no8zsJuAvgSs9e6+RrgHKEtbHB21Zz8wKiIfCQ+7+m7Dr6QNzgGvN7GNAMTDYzB5098+GXFdK6D6GEJnZNqDc3TNlAq5TYmbzgR8Bl7p7Q9j19BYzyyd+cv1K4oGwGvhMwjPOs5LF/8pZAuxz96+GXU9fC0YM33D3vwy7llTROQbpC/8FDAKeNbO1ZvbTsAvqDcEJ9luBlcRPwD6W7aEQmAPcCFwR/O+7NvhLWjKURgwiItKJRgwiItKJgkFERDpRMIiISCcKBhER6UTBICIinSgYRESkEwWDiIh0omAQEZFO/j9lj1R26+HtiAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import torch\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "x = torch.arange(-5., 5., 0.1)\n",
    "y = torch.tanh(x)\n",
    "\n",
    "plt.plot(x.numpy(), y.detach().numpy())\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Example 3-4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAD8CAYAAABXe05zAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAFnZJREFUeJzt3XeYlPXVxvH7SBFporIiUlwRC4iUZQXssQSxv5qoNPNaMRRFY48aTd7EJMYaRQ2JMUYWUMQWe8NetwDSBQQBwV2kC8u28/6xO2ZVYGd3yjPPzPdzXVwu7jh7BvCeH2ee2dvcXQCA8Ngp6AEAAPVDcANAyBDcABAyBDcAhAzBDQAhQ3ADQMgQ3AAQMgQ3AIQMwQ0AIdM4EXfatm1bz87OTsRdA0BaKigoWO3uWdHcNiHBnZ2drfz8/ETcNQCkJTNbGu1tWZUAQMhEdeI2syWSNkqqlFTh7rmJHAoAsH31WZUc6+6rEzYJACAqrEoAIGSiDW6X9KqZFZjZiG3dwMxGmFm+meWXlJTEb0IAwPdEG9xHunuOpJMkjTazo394A3cf7+657p6blRXVFS0AgAaIKrjdfUXNP4slPS2pXyKHAgBsX53BbWYtzKxV5GNJAyXNSvRgABAmn3yxRg+/94WSUQcZzVUl7SQ9bWaR209095cTOhUAhEjJxq0aM7FQLXZurCH9Oql504S8t/E7dd67uy+W1CuhUwBASFVWucZOLtL6LeV69MJ+CQ9tKUFveQeATHHP6wv0waJvdPvPe6pb+9ZJ+Zpcxw0ADfTW/GLd9+ZCnd23o87J7ZS0r0twA0ADfLVui658fLoO2quVfndGj6R+bYIbAOqprKJKoycWqrzS9cCwHO3StFFSvz47bgCopz+/PE9FX67TuKE56pLVMulfnxM3ANTDy7NW6uH3vtD5h2frlJ7tA5mB4AaAKC1Z/a2umTJTvTq10a9P7hbYHAQ3AEShtLxSo/IKtdNOpnFD+6hp4+Dikx03AETh1udma87KDXrk/EPVcbfmgc7CiRsA6jC1YLkmf7pMo4/dT8cetGfQ4xDcALAj81dt1I3PfKYBXXbXlSccEPQ4kghuANiuTVsrNDKvQK2aNdFfh/RR40apEZmpMQUApBh31/VTZ2rJ6m/118F9tGerZkGP9B2CGwC24bGPlur5mSt11cADddh+ewQ9zvcQ3ADwAzOWrdP/PT9Hxx6YpZHH7Bf0OD9CcANALes2l2lUXqH2bNVMd53TWzvtZEGP9CNcxw0ANaqqXFc9MUPFG0s15ZeHa7cWTYMeaZs4cQNAjb+9s1hvzCvWjSd3U+9ObYIeZ7sIbgCQ9PHib3THq/N1yiHt9b+HZwc9zg4R3AAyXvHGUo2ZVKTOuzfXn352iGrK0VMWO24AGa2yyjV20nRt2FKuf1/YT62aNQl6pDoR3AAy2t2vLdCHi7/RX5JY9hsrViUAMta0+cW6f9pCnZPbUWcnsew3VgQ3gIy0IsCy31gR3AAyTllFlUbnFaqi0vXg8L5q1iS5Zb+xYscNIOP88aW5mr5snR4YlqN927YIepx648QNIKO89NlKPfL+Ep1/eLZOPiSYst9YEdwAMsYXq7/VNU/OVO+Ay35jRXADyAil5ZUaOaFAjRuZxg3LCbTsN1bsuAFkhFuena15qzbqkQsOVYc2uwQ9TkzC+5QDAFF6smC5Hs+vKfs9MPiy31hFHdxm1sjMiszs+UQOBADxNG/VBt2UYmW/sarPiXuspLmJGgQA4m3T1gqNyitMubLfWEX1KMyso6RTJP0jseMAQHzULvu9b0hqlf3GKtqnn3skXSupKoGzAEDcRMp+rznxIA3oklplv7GqM7jN7FRJxe5eUMftRphZvpnll5SUxG1AAKiv6TVlv8cftKcuPbpL0OPEXTQn7iMknW5mSyRNlnScmU344Y3cfby757p7blZWVpzHBIDorNtcptE1Zb93ntMrJct+Y1VncLv7De7e0d2zJQ2W9Ka7D0/4ZABQT1VVrl/VlP0+MCxHbZqnZtlvrNLjJVYAkPTQO4v05rxi3XRKd/VK4bLfWNXrnZPu/paktxIyCQDE4KPF3+iOV+brlJ7t9YvD9gl6nITixA0g9Io3luqySUXK3qOF/vyznilf9hsrvlcJgFCLlP1uLC3XYxf1U8ud0z/W0v8RAkhrtct+D9orHGW/sWJVAiC0wlr2GyuCG0AoRcp+u7VvHbqy31gR3ABCp3bZ7wPDckJX9hsrdtwAQue2F6vLfh8MadlvrDhxAwiVF2au1L8+WKILjsjWSSEt+40VwQ0gNBaXbNJ1U2eqT+c2uuGk8Jb9xorgBhAKpeWVGpVXqCaNTOOGhrvsN1bsuAGEwm+enaX5X2/UI+cfqr1DXvYbq8x9ygIQGlPyl+mJ/OUac2xX/SQNyn5jRXADSGnzVm3Qzc/O0uH77aEr0qTsN1YEN4CUtbG0XCMnFKp1sya6d3AfNUrDUoSGYMcNICW5u65/6jN9uWazJl7cX1mtdg56pJTBiRtASnr0gyV6YeZKXT3wQPVPs7LfWBHcAFJO0Zdr9YcX56Zt2W+sCG4AKWXtt2UaM7Eorct+Y8WOG0DKqC77na6SjVs15ZeHpW3Zb6w4cQNIGQ++vUjT5pfoplO7pXXZb6wIbgAp4cNF3+jOV+frtF5767wB6V32GyuCG0Dgviv7bdtCfzzrkLQv+40VO24AgaqorNLlk4q0aWu58i7unxFlv7HiVwhAoO5+fYE+WrxGd57dSwfu1SrocUKBVQmAwEybV6xx0xZp8KGd9LO+HYMeJzQIbgCBWLFui658Yrq6t2+tW08/OOhxQoXgBpB0ZRVVGpVXqMoMLfuNFTtuAEl324tzNWPZOj00PEfZGVj2GytO3ACSKlL2e+ER+2pQj8ws+40VwQ0gaWqX/V5/0kFBjxNaBDeApNhSRtlvvNT5K2dmzczsEzObYWazzey3yRgMQHqJlP3efW7vjC/7jVU0L05ulXScu28ysyaS3jOzl9z9owTPBiBNPJG/TFMKluvy4yj7jYc6g9vdXdKmmp82qfnhiRwKQPqYu3KDbn5mlo7ouofGUvYbF1EtmcyskZlNl1Qs6TV3/zixYwFIBxtLyzUqr1C77tJE95xL2W+8RBXc7l7p7r0ldZTUz8x6/PA2ZjbCzPLNLL+kpCTecwIIGXfX9VOry37vG9KHst84qtfLuu6+TtI0SYO28bnx7p7r7rlZWVnxmg9ASD36wRK98NlKXXMiZb/xFs1VJVlm1qbm410k/VTSvEQPBiC8ImW/J3TbUyOOouw33qK5qqS9pEfNrJGqg/4Jd38+sWMBCKtI2W+71s1059m9KftNgGiuKpkpqU8SZgEQclVVritryn6fHHmYdm3eJOiR0hJvXQIQNw++vUhvzS/Rzad1V8+OlP0mCsENIC4+WLRad746X6f32lvD+3cOepy0RnADiFnxhlJdPmm69qXsNyn4ftwAYlJRWaXLJhXp260VmnhJf7Wg7Dfh+BUGEJO7Xlugj79Yo7vO6aUD2lH2mwysSgA02JvzvtYDby3SkH6ddFYOZb/JQnADaJDlazfrysdnqHv71rrlNMp+k4ngBlBvWysqNTqvUFVVlP0GgR03gHq77YW5mrF8PWW/AeHEDaBe/jPjKz364VJddCRlv0EhuAFEbVHJJl0/daZyKPsNFMENICpbyio1akKhmjbeSfcPzVGTRsRHUNhxA6iTu+umZ2ZpQfFG/euCfpT9BoynTAB1eiJ/maYWLtdlx3bVMQdQlBI0ghvADs35aoN+8+xsyn5TCMENYLs2lJZrVF6B2jRvonsHU/abKthxA9gmd9d1T87UsrVbNHnEALVtSdlvquDEDWCbHnl/iV6atUrXnnigDs3ePehxUAvBDeBHCr9cq9tenKsTurXTiKMp+001BDeA71n7bZnG5BVqr12b6c6ze1GKkILYcQP4TlWV64rHp2v1pjJNHXk4Zb8pihM3gO+Mm7ZQby8o0W9O665DOu4a9DjYDoIbgCTpg4WrdffrC3RG7701jLLflEZwA9DXG0p1+eQi7du2hW47k7LfVMeOG8hw/y37rdTESwZQ9hsC/A4BGe6OVxfoE8p+Q4VVCZDB3pj7tR56m7LfsCG4gQy1bM1m/eoJyn7DiOAGMtDWikqNnlioKnc9OJyy37Bhxw1koD+8MFczl6/XQ8P7ap89KPsNG07cQIb5z4yv9O8Pl+riI/fVoB57BT0OGoDgBjJIpOy37z676TrKfkOrzuA2s05mNs3M5pjZbDMbm4zBAMTX5rIKjZxQoJ2bNNL9Q/tQ9hti0ey4KyRd5e6FZtZKUoGZvebucxI8G4A4iZT9fl68SY9e0E/td6XsN8zqfMp195XuXljz8UZJcyV1SPRgAOLn8U+X6anCFbrsuP11NGW/oVevvyuZWbakPpI+3sbnRphZvpnll5SUxGc6ADGb/dV6/ea52Tqya1uNPX7/oMdBHEQd3GbWUtJUSVe4+4Yfft7dx7t7rrvnZmXxjA6kguqy30Lt1ryJ7hncm7LfNBHVddxm1kTVoZ3n7k8ldiQA8eDuunbKTC2n7DftRHNViUl6WNJcd78r8SMBiId/vr9EL89epesGUfabbqJZlRwh6TxJx5nZ9JofJyd4LgAxKFi6Vn98ca5+2r2dLjmKst90U+eqxN3fk8RiDAiJNd+WaczEQrVv00x3UPablvheJUAaiZT9fhMp+92Fst90xFungDQybtpCvUPZb9ojuIE0Qdlv5iC4gTQQKfvtktWSst8MwI4bCLnaZb+TLsmh7DcD8DsMhFyk7Pfuc3tpf8p+MwKrEiDEXp8TKfvtrDP7UPabKQhuIKSqy36nq0eH1rrltO5Bj4MkIriBEIqU/bqkB4b2pew3w7DjBkLo989Xl/2OP6+vOu/RPOhxkGScuIGQeW7GV3rso6UacXQXDTyYst9MRHADIbKwuLrs99Ds3XTNiQcGPQ4CQnADIbG5rEKj8gq0S5NGum9IDmW/GYwdNxACtct+H7uwv/batVnQIyFAPGUDIRAp+x17/P46cv+2QY+DgBHcQIqLlP0etX9bXXYcZb8guIGUFin73b15U91zLmW/qMaOG0hRtct+Hx8xQHtQ9osanLiBFPXwe1/o5dmrdP2gg5RL2S9qIbiBFFSwdI3+9NI8DezeThcftW/Q4yDFENxAiqku+y3S3m120V8o+8U2sOMGUsh3Zb/flukpyn6xHZy4gRRyf03Z762nHaweHSj7xbYR3ECKeL+m7PfMPh00pF+noMdBCiO4gRTw9YZSjZ1cpK5ZLfWHM3uw18YOseMGAlZeWaUxEwu1uaxSk0fkqHlT/rfEjvEnBAjYHa/M16dL1urewb3VdU/KflE3ViVAgF6b87X+9s5iDevfWWf07hD0OAgJghsIyLI1m3VVTdnvzadS9ovoEdxAAErLKzUqj7JfNAw7biAAv39hjj5bQdkvGqbOE7eZ/dPMis1sVjIGAtLds9NXaMJHX1L2iwaLZlXyL0mDEjwHkBEWFm/UDU99RtkvYlJncLv7O5LWJGEWIK1tLqvQyAmFlP0iZuy4gSRwd9309CwtLKHsF7GL21O+mY0ws3wzyy8pKYnX3QJpYfKny/RUEWW/iI+4Bbe7j3f3XHfPzcrKitfdAqE3a8V63VJT9ns5Zb+IA5ZsQAJtKC3X6In/LfvdibJfxEE0lwNOkvShpAPNbLmZXZT4sYDwi5T9rli7ReOG9aHsF3FT54uT7j4kGYMA6SZS9nvTKd3Udx/KfhE/rEqABIiU/Z54cDtddCRlv4gvghuIs282bdXovOqy39t/Ttkv4o/ruIE4qqwp+12zmbJfJA4nbiCO7n9zod79fDVlv0goghuIk/c+X6173ligsyj7RYIR3EAcrFpfXfa7/54t9XvKfpFgBDcQo/LKKl02qVBbyiv1wDDKfpF4/AkDYkTZL5KNEzcQg0jZ7/ABlP0ieQhuoIEiZb+HdNiVsl8kFcENNECk7FeSHhiWo50bU/aL5GHHDTRApOz377/IVafdKftFcnHiBuopUvZ76dFd9NPu7YIeBxmI4AbqoXbZ79WU/SIgBDcQpUjZb/OmjXT/UMp+ERx23EAU3F031pT9Triov9q1puwXweHIAERh0ifL9HTRCl15wgE6oitlvwgWwQ3UYdaK9br1P7N19AFZGnNs16DHAQhuYEfWbynXqLxC7dGCsl+kDnbcwHa4u66ZMkNfrduixy8doN1bNA16JEASJ25gux5+7wu9OudrXX/SQZT9IqUQ3MA2RMp+Bx28F2W/SDkEN/ADkbLfDrvtotvP7kkpAlIOO26glh+W/bZuRtkvUg8nbqCW+978XO9+vlq/PZ2yX6Qughuo8e7nJbr3jc91Vk4HDT6Usl+kLoIbkLRy/RZdMXl6ddnv/1D2i9RGcCPjlVdWaczEopqy376U/SLl8ScUGe/2l+epYOla/XVIH3Xds2XQ4wB14sSNjPbK7FX6+7tf6LwB++j0XnsHPQ4QFYIbGevLbzbr6ikz1LPjrrrp1G5BjwNELargNrNBZjbfzBaa2fWJHgpItNLySo3MK5BJGjeUsl+ES53BbWaNJI2TdJKk7pKGmFn3RA8GJNLvnp+j2V9t0F3n9KbsF6ETzYm7n6SF7r7Y3cskTZZ0RmLHAhLnmaIVmvjxl7r0mC46gbJfhFA0V5V0kLSs1s+XS+qfiGFOu+89lZZXJuKuge8sXbNZ/bJ31zUDKftFOMXtckAzGyFphCR17ty5QfexX1YLlVVWxWskYJtyOu+mqwYeoMaU/SKkognuFZJqv/+3Y82/+x53Hy9pvCTl5uZ6Q4a5Z3CfhvxnAJBRojlyfCppfzPb18yaShos6bnEjgUA2J46T9zuXmFmYyS9IqmRpH+6++yETwYA2Kaodtzu/qKkFxM8CwAgCrw6AwAhQ3ADQMgQ3AAQMgQ3AIQMwQ0AIWPuDXqvzI7v1KxE0tK433HitZW0OughkiwTH7OUmY+bx5za9nH3rGhumJDgDiszy3f33KDnSKZMfMxSZj5uHnP6YFUCACFDcANAyBDc3zc+6AECkImPWcrMx81jThPsuAEgZDhxA0DIENzbYGZXmZmbWdugZ0kGM/uLmc0zs5lm9rSZtQl6pkTJxOJrM+tkZtPMbI6ZzTazsUHPlCxm1sjMiszs+aBniSeC+wfMrJOkgZK+DHqWJHpNUg937ylpgaQbAp4nITK4+LpC0lXu3l3SAEmjM+RxS9JYSXODHiLeCO4fu1vStZIyZvnv7q+6e0XNTz9SdctROsrI4mt3X+nuhTUfb1R1kHUIdqrEM7OOkk6R9I+gZ4k3grsWMztD0gp3nxH0LAG6UNJLQQ+RINsqvk77AKvNzLIl9ZH0cbCTJMU9qj6EpV2RbdzKgsPCzF6XtNc2PnWjpF+rek2Sdnb0uN392Zrb3Kjqv1bnJXM2JIeZtZQ0VdIV7r4h6HkSycxOlVTs7gVm9pOg54m3jAtudz9hW//ezA6RtK+kGWYmVa8LCs2sn7uvSuKICbG9xx1hZudLOlXS8Z6+14hGVXydjsysiapDO8/dnwp6niQ4QtLpZnaypGaSWpvZBHcfHvBcccF13NthZksk5bp7WL5BTYOZ2SBJd0k6xt1Lgp4nUcyssapffD1e1YH9qaSh6d6hatUnkUclrXH3K4KeJ9lqTtxXu/upQc8SL+y4IUn3S2ol6TUzm25mDwU9UCLUvAAbKb6eK+mJdA/tGkdIOk/ScTW/v9NrTqIIKU7cABAynLgBIGQIbgAIGYIbAEKG4AaAkCG4ASBkCG4ACBmCGwBChuAGgJD5fxQePt3dlUjmAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import torch\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "relu = torch.nn.ReLU()\n",
    "x = torch.arange(-5., 5., 0.1)\n",
    "y = relu(x)\n",
    "\n",
    "plt.plot(x.numpy(), y.detach().numpy())\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAD8CAYAAABjAo9vAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAHkxJREFUeJzt3XtUVeeB9/Hvw00EucjFBEVEgVYxXqIIKrm00U5jksY1Y9Kk3mK85U0n76tZfSdN2jdr0mZ1VjrtqplO0zVRQlK81EzNNG1t2pnodGqigoIXGoOJQKXilYsIigie87x/QF1Ja6LIhn3OPr/PWllROevZvy3w82Hv5+zHWGsRERHvCHM7gIiIOEvFLiLiMSp2ERGPUbGLiHiMil1ExGNU7CIiHqNiFxHxGBW7iIjHqNhFRDwmwo2DpqSk2MzMTDcOLSIStCoqKhqttanXep0rxZ6ZmUl5ebkbhxYRCVrGmLrreZ0uxYiIeIwjM3ZjzFGgDfABl621eU6MKyIivefkpZjPW2sbHRxPRERugC7FiIh4jFPFboH/MsZUGGNWOjSmiIjcAKcuxdxmrT1ujBkGvG2MOWyt3fHRF/QU/kqAjIwMhw4rIiJ/yZEZu7X2eM//zwA/B/Kv8pq11to8a21eauo1l2GKiMgN6nOxG2NijTFxf/418DfAe30dV0TES9o7L/PcLw9xrr2r34/lxIz9JuBdY8xBYA/wa2vtbx0YV0TEEzq6fDy2voKS3UfZ96ez/X68Pl9jt9bWApMcyCIi4jldPj9PbNrHO0ca+f6Dk/j82GH9fkwtdxQR6Sc+v+XJ1w+wreoMz88dzwNT0wfkuCp2EZF+4Pdbvv5GJVsrT/LMnLEsmpE5YMdWsYuIOMxayz/+8hBbKupZNSuHx+7MGtDjq9hFRBxkreWF3xxmfWkdK+8Yw+rZOQOeQcUuIuKgf9l+hJd31LJo+iiemTMWY8yAZ1Cxi4g4ZO2OGl7cdoR5U9L51v3jXSl1ULGLiDhifWkd//TWYe6dmMY/PzCRsDB3Sh1U7CIifbalop5n33yP2eOG8eJDkwl3sdRBxS4i0idbK0/w1JaD3Jadwo/mTyEy3P1adT+BiEiQ2l51mtWbDzB11FDWLp5KdGS425EAFbuIyA1590gjj2/YR+7weIqXTCMmyskN6fpGxS4i0kt7jzazoqScMamxlCzNJy460u1IH6NiFxHphYPHWnj01b2kJUSzflkBiTFRbkf6Kyp2EZHrdPhUK4+8uoehsZFsXFFAatwgtyNdlYpdROQ61DScZ2FRGdER4WxaPp20hMFuR/pEKnYRkWs41tzOgnVlWAsblhcwMinG7UifKnBu44qIBKBT5zqYX1TKxS4fm1dOJ3vYELcjXZNm7CIin6Ch7RLzi0o5e6GLkqX5jEuLdzvSdVGxi4hcRUt7J4teKeNEy0WKl0xj0shEtyNdNxW7iMhfaOvo4pHiPdQ2XGDd4jzyRye5HalXVOwiIh/R3nmZZa+Vc+hEKz9eMIXbc1LdjtRrKnYRkR4dXT4eW19BeV0zLz48mdm5N7kd6YZoVYyICNDl8/PEpv28c6SR7z84ifsmDnc70g3TjF1EQp7Pb3ny9QNsqzrN83PH88DUdLcj9YljxW6MCTfG7DfGbHVqTBGR/ub3W77+RiVbK0/yjXvGsmhGptuR+szJGfsqoMrB8URE+pW1ln/85SG2VNSzalYOK+/IcjuSIxwpdmNMOnAvUOTEeCIi/c1aywu/Ocz60jpW3jGG1bNz3I7kGKdm7C8CTwH+T3qBMWalMabcGFPe0NDg0GFFRG7Mv2w/wss7alk4PYNn5ozFGHf3KXVSn4vdGHMfcMZaW/Fpr7PWrrXW5llr81JTg29dqIh4x9odNby47QjzpqTz7ftv8VSpgzMz9kLgfmPMUWAzcJcxZoMD44qIOG797qP801uHuXdCGt+dN4GwMG+VOjhQ7NbaZ6y16dbaTOBh4L+ttQv7nExExGE/Kz/Gs784xOxxw1jz0GQiwr254tubZyUi8he2Vp7g629Uclt2Cj+aP4WoCO/Wn6PvPLXW/g/wP06OKSLSV9urTrN68wGmjhrK2sVTiY4MdztSv/LuP1kiIsC7Rxp5fMM+cofHU7xkGjFR3n+SiopdRDxr79FmVpSUMyY1lpKl+cRFR7odaUCo2EXEkyrrW3j01b2kJUazflkBiTFRbkcaMCp2EfGcw6daWVy8h6GxkWxcXkBq3CC3Iw0oFbuIeEpNw3kWFpURHRHOpuXTSUsY7HakAadiFxHPONbczoJ1ZQBsXFHAyKQYlxO5w/u3h0UkJJw618H8olIudvnYvHI6WalD3I7kGs3YRSToNbRdYn5RKWcvdFGyNJ9xafFuR3KVil1EglpLeyeLXinjRMtFipdMY9LIRLcjuU7FLiJBq62ji0eK91DbcIF1i/PIH53kdqSAoGIXkaDU3nmZZa+Vc+hEKy8tmMLtOXoc+J+p2EUk6HR0+XhsfQXldc28+PBkvpB7k9uRAopWxYhIUOny+Xli037eOdLI9x6YyH0Th7sdKeBoxi4iQcPntzz5+gG2VZ3m+bnjeTBvpNuRApKKXUSCgt9v+foblWytPMkzc8ayaEam25EClopdRAKetZbnfnWILRX1rJqVw2N3ZrkdKaCp2EUkoFlreeG3hynZXcdjd4xh9ewctyMFPBW7iAS0H26v5uXf17Jo+iienjMWY7y3+bTTVOwiErDW7ahlzbYPeWBqOt+6f7xK/Tqp2EUkIG0oreM7b1Vx78Q0vjtvImFhKvXrpWIXkYDzRkU9/+/N95g9bhgvPjSZcJV6r6jYRSSg/LryJP+w5SC356Two/lTiAxXTfWW/sZEJGBsrzrNqs37mTpqKC8vmkp0ZLjbkYJSn4vdGBNtjNljjDlojDlkjPmWE8FEJLS8e6SRxzfuI3d4PMVLphETpSee3Cgn/uYuAXdZa88bYyKBd40xv7HWljowtoiEgL1Hm1lRUs6YlFhKluYTFx3pdqSg1udit9Za4HzPbyN7/rN9HVdEQkNlfQuPvrqXtMRo1i8rIDEmyu1IQc+Ra+zGmHBjzAHgDPC2tbbsKq9ZaYwpN8aUNzQ0OHFYEQlyh0+1srh4D4kxkWxcXkBq3CC3I3mCI8VurfVZaycD6UC+MeaWq7xmrbU2z1qbl5qqB+KLhLqahvMsLCojOiKcn66YTlrCYLcjeYajq2KstS3A74C7nRxXRLzlWHM7C9aVYS1sWF7AyKQYtyN5ihOrYlKNMYk9vx4MfAE43NdxRcSbTp3rYH5RKRe7fGxYXkD2sCFuR/IcJ1bFpAE/McaE0/0Pxb9ba7c6MK6IeEzj+UssKCrl7IUuNi4vYFxavNuRPMmJVTGVwK0OZBERD2tp72RhURnHWy5SsrSASSMT3Y7kWXrnqYj0u7aOLh4p3kNtwwXWLc4jf3SS25E8TcUuIv2qvfMyy14r59CJVl5aMIXbc7Qqrr+p2EWk33R0+XhsfQXldc2seWgyX8i9ye1IIUEPYxCRftHl8/PEpv28c6SR7z84iS9NGu52pJChGbuIOM7ntzz5+gG2VZ3m+bnjeWBqutuRQoqKXUQc5fdbvv5GJVsrT/KNe8ayaEam25FCjopdRBxjreW5Xx1iS0U9q2blsPKOLLcjhSQVu4g4wlrLC785TMnuOlbeMYbVs3PcjhSyVOwi4ogfbq/m5R21LJo+imfmjMUY7VPqFhW7iPTZuh21rNn2IQ9MTedb949XqbtMxS4ifbK+tI7vvFXFvRPT+O68iYSFqdTdpmIXkRu2paKeZ998j1ljh7Hmy5MJV6kHBBW7iNyQrZUneGrLQW7LTuGlBVOIilCdBAp9JkSk17a9f5rVmw8wddRQ1i6eSnRkuNuR5CNU7CLSK+8eaeSrG/eROzyeV5ZMIyZKTyYJNCp2Eblue482s6KknDGpsZQszSc+OtLtSHIVKnYRuS4Hj7Xw6Kt7SUuIZv2yAhJjotyOJJ9AxS4i11R1spXFxXtIjIlkw/ICUuMGuR1JPoWKXUQ+VU3DeRa9UsbgyHA2LZ/O8MTBbkeSa1Cxi8gnOtbczoJ1ZVgLG5YXkJEc43YkuQ66nS0iV3Xy3EXmF5VyscvHT1dMJ3vYELcjyXXSjF1E/kpD2yUWFJVx9kIXJUvzyR0e73Yk6QUVu4h8TEt7J4teKeNEy0WKl0xj0shEtyNJL/W52I0xI40xvzPGvG+MOWSMWeVEMBEZeG0dXTxSvIfahgusW5xH/ugktyPJDXDiGvtl4GvW2n3GmDigwhjztrX2fQfGFpEB0t55mWWvlXPoRCv/tnAqt+ekuh1JblCfZ+zW2pPW2n09v24DqoARfR1XRAZOR5ePx9ZXUF7XzJqHJjM79ya3I0kfOHqN3RiTCdwKlF3lYyuNMeXGmPKGhgYnDysifdDl8/PEpn28c6SRF+ZN5EuThrsdSfrIsWI3xgwB3gBWW2tb//Lj1tq11to8a21eaqp+xBMJBD6/5cnXD7Ct6gzPzx3Pl/NGuh1JHOBIsRtjIuku9Y3W2v9wYkwR6V9+v+XpNyrZWnmSZ+aMZdGMTLcjiUOcWBVjgFeAKmvtD/oeSUT6m7WW5351iJ9V1LNqVg6P3ZnldiRxkBMz9kJgEXCXMeZAz3/3ODCuiPQDay0v/PYwJbvrWHH7aFbPznE7kjisz8sdrbXvAtroUCRI/HB7NS//vpaF0zP4xj3j6P6hW7xE7zwVCSHrdtSyZtuHzJuSzrfvv0Wl7lEqdpEQsb60ju+8VcW9E9L47rwJhIWp1L1KxS4SArZU1PPsm+8xa+ww1jw0mYhwfet7mT67Ih7368qTPLXlIIXZyby0YApREfq29zp9hkU8bNv7p1m1eT9TMoaybnEe0ZHhbkeSAaBiF/God4808tVN+8gdHk/xo9OIidK+OqFCxS7iQeVHm1lRUs7o5Fh+8mg+8dGRbkeSAaRiF/GYyvoWlry6l7SEaDYsL2BobJTbkWSAqdhFPOTwqVYWF+8hMSaSjSsKSI0b5HYkcYGKXcQjahrOs7CojOiIcDYtn05awmC3I4lLVOwiHnCsuZ2FRd3bIGxcUUBGcozLicRNuk0uEuROnetgflEp7Z0+Nq+cTlbqELcjics0YxcJYo3nL7GgqJSzF7ooWZrPuLR4tyNJAFCxiwSplvZOFhaVcbzlIq8+Oo1JIxPdjiQBQsUuEoTaOrp4pHgPtQ0XWLc4j2mZSW5HkgCiYhcJMhc7fSx7rZxDJ1r58YIp3J6jPYTl41TsIkHk0mUfK9eXU17XzIsPT2Z27k1uR5IApFUxIkGiy+fn7zfu550jjXzvgYncN3G425EkQGnGLhIEfH7Lk68fYFvVaZ6fO54H80a6HUkCmIpdJMD5/Zan36hka+VJnpkzlkUzMt2OJAFOxS4SwKy1PPerQ/ysop5Vs3J47M4styNJEFCxiwQoay0v/PYwJbvrWHnHGFbPznE7kgQJFbtIgPrh9mpe/n0ti6aP4pk5YzFGm0/L9XGk2I0xxcaYM8aY95wYTyTUrdtRy5ptHzJvSjrfun+8Sl16xakZ+2vA3Q6NJRLS1pfW8Z23qrh3QhrfnTeBsDCVuvSOI8Vurd0BNDsxlkgo21JRz7NvvsesscNY89BkIsJ1tVR6T181IgHi15UneWrLQW7LTuGlBVOIitC3p9yYAfvKMcasNMaUG2PKGxoaBuqwIkFhe9VpVm3ez9RRQ1m7eCrRkeFuR5IgNmDFbq1da63Ns9bmpabqoUUif/bukUYe37iP3OHxFC+ZRkyUnvQhfaOf9URcVH60mRUl5YxJiaVkaT5x0ZFuRxIPcGq540+B3cBnjTH1xphlTowr4mWV9S08+upe0hKiWb+sgMSYKLcjiUc48jOftfYrTowjEioOn2plcfEeEmMj2biigNS4QW5HEg/RpRiRAVbbcJ6FRXuIjghn0/LppCUMdjuSeIyKXWQAHWtuZ0FRGdZaNiwvYGRSjNuRxIN0+11kgJw618H8olLaO31sXjmd7GFD3I4kHqUZu8gAaDx/iQVFpZy90EXJ0nzGpcW7HUk8TMUu0s9a2jtZ9MoejrdcpHjJNCaNTHQ7knicil2kH7V1dPFI8R5qzpxn3eI88kcnuR1JQoCKXaSfXOz0sey1cg6daOXHC6Zwe47ecS0DQ8Uu0g86unysXF9OeV0zax6azOzcm9yOJCFEq2JEHNbl8/PEpv28c6SR7z0wkS9NGu52JAkxmrGLOMjntzz5+gG2VZ3m+bnjeTBvpNuRJASp2EUc4vdbnn6jkq2VJ3lmzlgWzch0O5KEKBW7iAOstTz3q0P8rKKe/zMrh8fuzHI7koQwFbtIH1lreeG3hynZXceK20fz5OwctyNJiFOxi/TRv/53NS//vpaF0zP4xj3jMEabT4u7VOwifbBuRy0/ePtD/m7KCL59/y0qdQkIKnaRG7S+tI7vvFXFvRPS+Od5EwkLU6lLYFCxi9yALRX1PPvme8waO4w1D00mIlzfShI49NUo0ktbK0/w1JaD3JadwksLphAVoW8jCSz6ihTphe1Vp1m9+QBTRw1l7eKpREeGux1J5K+o2EWu087qRh7fuI/c4fEUL5lGTJSeyCGBScUuch32Hm1m+U/KGZMSS8nSfOKiI92OJPKJVOwi11BZ38Kjr+4lLSGa9csKSIyJcjuSyKdSsYt8isOnWllcvIfEmEg2riggNW6Q25FErsmRYjfG3G2M+cAYU22MedqJMUXcVttwnoVFe4iOCGfT8umkJQx2O5LIdelzsRtjwoGXgDlALvAVY0xuX8cVcdOx5nYWFJVhrWXD8gIykmPcjiRy3ZyYsecD1dbaWmttJ7AZmOvAuCKuOHWug/lFpbR3+tiwvIDsYUPcjiTSK06s1xoBHPvI7+uBAgfGFRkQ1lpqGi6wq6aRndWN7KppwlrYuLyAcWnxbscT6bUBW4hrjFkJrATIyMgYqMOKXNXJcxfZWd3Erp4iP9XaAcCIxMHcPf5mFs/IZEJ6gsspRW6ME8V+HPjo/l/pPX/2MdbatcBagLy8POvAcUWuW0t7J7trmthZ08iu6iZqGy8AkBQbxYysZAqzUpiZlcyo5Bg9oVGCnhPFvhfIMcaMprvQHwbmOzCuyA1r77zMnj82s6umiZ3Vjbx/shVrITYqnPzRScwvyGBmVgpjb47TUxnFc/pc7Nbay8aYJ4D/BMKBYmvtoT4nE+mFLp+fA8dauq+RVzex/9hZunyWqPAwJmcksnrWZyjMTmbSyEQi9SRG8ThHrrFba98C3nJiLJHr4fdb3j/Zyq6a7mvke/7YTHunD2NgwogElt42msKsFKZlJjE4Sg/qktCipxhJULDWcrSpvWfVSiO7a5o4294FQFZqLPOmpFOYncz0Mcl6y7+EPBW7BKwzrR3srGlkZ3UTu2uaON5yEYCb46P5/NhhFGalUJidws0J0S4nFQksKnYJGOcudlFa270EcWdNE9VnzgOQGBPJjDHJ/K/PZVGYlczolFitXBH5FCp2cU1Hl4+9R5u715PXNPLe8XP4LQyODGfa6CQenJpOYXYKuWnxWrki0gsqdhkwl31+Dtaf65mRN7KvroVOn5+IMMOtGYn877tymJmVzK0ZQ7XdnEgfqNil3/j9lg9Ot7GzuvtmZ9kfmzl/6TIAuWnxPDJzFDOzU8jPTCJ2kL4URZyi7yZx1LHm7pUrO2ua2F3TSOP5TgAyk2O4f/JwCrNSmJGVTFKsVq6I9BcVu/RJQ9ul7rXk1d1v168/271yZVjcIG7LTmFmdvfKlRGJepa5yEBRsUuvtHV0UVbbfOWZKx+cbgMgPjqC6WOSWXH7GAqzk8lKHaKVKyIuUbHLp+ro8rHvT2evzMgr68/h81sGRYQxLTOJubd2X165ZUQC4Vq5IhIQVOzyMT6/5Q/Hz1254bn3aDOXLvsJDzNMSk/gq5/LYmZWCrdmJBIdqbfqiwQiFXuIs9ZSfeb8lRuepbVNtHV0r1wZe3McCwpGUZidTP7oJOKiI11OKyLXQ8UegurPtl+5tLKrpomGtksAjEwazL0T0piZ3f1s8pQhg1xOKiI3QsUeAprOX2J3bdOVd3jWNbUDkDIkihlZKRRmJVOYncLIJG3YLOIFKnYPunCpe5OJP19eqTrZCkDcoAgKxiTxyIxMCrNT+MxNWrki4kUqdg+4dNnH/j+1XHl41sFjLVz2W6IiwsgbNZR/+OJnmZmVzIQRCURokwkRz1OxByGf3/L+idaeR9o2svdoMx1dfsIMTEhPZOUdYyjMTmHqqKFauSISglTsQcBaS03DBXb1FHlpbTPnLnZvMpEzbAgPT8tgZlYyBWOSSRislSsioU7FHqBOnrt45WbnruomTrV2ADAicTBfHH8ThdkpzBiTzLB4bTIhIh+nYg8QZy90UlrbdOWt+rWNFwBIio1iRlZyz25ByWQkxeiGp4h8KhW7S9o7u1eu7KppYmd1I++fbMVaiI0KJ390EvMLMpiZlcLYm+O0yYSI9IqKfYB0+fwcONbSvRlzdRP7j52ly2eJCg9jckYiq2d9hsLsZCaNTCRSK1dEpA9U7P3E77dUnWq98g7PPX9spr3ThzFwy/AElt42msKsFKZlJjE4SitXRMQ5fSp2Y8yDwHPAOCDfWlvuRKhgZK3laFP7lZudu2ubaL7QvcnEmNRY5k1JpzA7meljkkmM0SYTItJ/+jpjfw/4O+BlB7IEndOtHT1LEJvYVd3IiXPdK1fSEqL5/GeHMTMrmZnZyaQlaJMJERk4fSp2a20VEDKrNM5d7KK0tunKOzyrz5wHIGFwJDOzknn8893PXRmdEhsyfyciEnh0jf1TXOz0UV7XvXJlV3Ujfzh+Dr+FwZHhTBudxINT0ynMTiE3LV4rV0QkYFyz2I0x24Cbr/Khb1prf3G9BzLGrARWAmRkZFx3wIF02efnYP25nhl5I/vqWuj0+YkIM9yakcgTd+VQmJXMrRlDiYrQyhURCUzXLHZr7WwnDmStXQusBcjLy7NOjNlX1lo+ON125Rp52R+bOX+pe5OJ3LR4Hpk5ipnZKeRnJhE7SD/ciEhwCLm2+lPPypWdNU3srmmk8Xz3ypXM5Bjun9y9f+eMrGSSYrVyRUSCU1+XO/4t8K9AKvBrY8wBa+0XHUnmkIa2S1eWIO6saaT+7EUAhsUN4rbsFGZmp1CYncKIRK1cERFv6OuqmJ8DP3coiyPaOrooq22+8syVD063ARAXHcGMMcmsuH0MhdnJZKVqkwkR8aagvxTT0eVjX93ZnmeTN/GH4+fw+S2DIsKYlpnE3Fu7L6/cMiKBcK1cEZEQEHTF7vNb/nD83JXLK3uPNnPpsp/wMMOk9AS++rksZmQlMyVDm0yISGgKqmL/4fYjrHunlraO7pUrY2+OY0HBKAqzk8kfnURctDaZEBEJqmK/OSGaeyekMTM7hZlZyaQMGeR2JBGRgBNUxf7lvJF8OW+k2zFERAKa3j4pIuIxKnYREY9RsYuIeIyKXUTEY1TsIiIeo2IXEfEYFbuIiMeo2EVEPMZYO/B7XhhjGoC6AT9w36UAjW6HGGCheM4QmucdiucMwXXeo6y1qdd6kSvFHqyMMeXW2jy3cwykUDxnCM3zDsVzBm+ety7FiIh4jIpdRMRjVOy9s9btAC4IxXOG0DzvUDxn8OB56xq7iIjHaMYuIuIxKvYbZIz5mjHGGmNS3M7S34wx3zPGHDbGVBpjfm6MSXQ7U38xxtxtjPnAGFNtjHna7TwDwRgz0hjzO2PM+8aYQ8aYVW5nGijGmHBjzH5jzFa3szhJxX4DjDEjgb8B/uR2lgHyNnCLtXYi8CHwjMt5+oUxJhx4CZgD5AJfMcbkuptqQFwGvmatzQWmA38fIucNsAqocjuE01TsN2YN8BQQEjcorLX/Za293PPbUiDdzTz9KB+ottbWWms7gc3AXJcz9Ttr7Ulr7b6eX7fRXXQj3E3V/4wx6cC9QJHbWZymYu8lY8xc4Li19qDbWVyyFPiN2yH6yQjg2Ed+X08IFNxHGWMygVuBMneTDIgX6Z6g+d0O4rSg2vN0oBhjtgE3X+VD3wS+QfdlGE/5tHO21v6i5zXfpPvH9o0DmU0GhjFmCPAGsNpa2+p2nv5kjLkPOGOtrTDGfM7tPE5TsV+FtXb21f7cGDMBGA0cNMZA9yWJfcaYfGvtqQGM6LhPOuc/M8YsAe4DZlnvrpE9Dnx0t/T0nj/zPGNMJN2lvtFa+x9u5xkAhcD9xph7gGgg3hizwVq70OVcjtA69j4wxhwF8qy1wfIAoRtijLkb+AFwp7W2we08/cUYE0H3zeFZdBf6XmC+tfaQq8H6memepfwEaLbWrnY7z0DrmbH/X2vtfW5ncYquscv1+BEQB7xtjDlgjPk3twP1h54bxE8A/0n3DcR/93qp9ygEFgF39Xx+D/TMZCVIacYuIuIxmrGLiHiMil1ExGNU7CIiHqNiFxHxGBW7iIjHqNhFRDxGxS4i4jEqdhERj/n/SqiBN7fnUAsAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import torch.nn as nn\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "prelu = nn.PReLU(num_parameters=1)\n",
    "x = torch.arange(-5., 5., 0.1)\n",
    "y = prelu(x)\n",
    "\n",
    "plt.plot(x.numpy(), y.detach().numpy())\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Example 3-6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[-2.0260, -2.0655, -1.2054]])\n",
      "tensor([[0.2362, 0.2271, 0.5367]])\n",
      "tensor([1.])\n"
     ]
    }
   ],
   "source": [
    "softmax = nn.Softmax(dim=1)\n",
    "x_input = torch.randn(1, 3)\n",
    "y_output = softmax(x_input)\n",
    "print(x_input)\n",
    "print(y_output)\n",
    "print(torch.sum(y_output, dim=1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Example 3-7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(1.6031, grad_fn=<MseLossBackward>)\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "\n",
    "mse_loss = nn.MSELoss()\n",
    "outputs = torch.randn(3, 5, requires_grad=True)\n",
    "targets = torch.randn(3, 5)\n",
    "loss = mse_loss(outputs, targets)\n",
    "loss.backward()\n",
    "print(loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Example 3-8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(2.5949, grad_fn=<NllLossBackward>)\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "\n",
    "ce_loss = nn.CrossEntropyLoss()\n",
    "outputs = torch.randn(3, 5, requires_grad=True)\n",
    "targets = torch.tensor([1, 0, 3], dtype=torch.int64)\n",
    "loss = ce_loss(outputs, targets)\n",
    "loss.backward()\n",
    "print (loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Example 3-9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[0.7411],\n",
      "        [0.5622],\n",
      "        [0.8286],\n",
      "        [0.6569]], grad_fn=<SigmoidBackward>)\n",
      "tensor(0.5958, grad_fn=<BinaryCrossEntropyBackward>)\n"
     ]
    }
   ],
   "source": [
    "bce_loss = nn.BCELoss()\n",
    "sigmoid = nn.Sigmoid()\n",
    "probabilities = sigmoid(torch.randn(4, 1, requires_grad=True))\n",
    "print(probabilities)\n",
    "\n",
    "targets = torch.tensor([1, 0, 1, 0], dtype=torch.float32).view(4, 1)\n",
    "loss = bce_loss(probabilities, targets)\n",
    "loss.backward()\n",
    "print(loss)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "nlpbook",
   "language": "python",
   "name": "nlpbook"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
